package com.zhanggm.s3.house.webmagic;

import org.springframework.beans.BeanUtils;
import us.codecraft.webmagic.*;
import us.codecraft.webmagic.pipeline.FilePipeline;
import us.codecraft.webmagic.pipeline.Pipeline;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Html;

import java.util.List;
import java.util.Map;


/**
 * 网易爬虫
 */
public class WangYiSpider implements PageProcessor {

    Site site = Site.me().setTimeOut(1000).setRetryTimes(3).setSleepTime(300)
            .setUserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36")
            .addCookie("","");

    @Override
    public void process(Page page) {
        Html html = page.getHtml();
        //获取抓取的超链接
        List<String> all = html.xpath("//div[@class='hidden']").links().all();
        page.addTargetRequests(all);
        //过滤内容，找符合条件的页面
        String url = page.getUrl().get();
        if (!url.startsWith("https://www.163.com/news/article/")) {
            page.setSkip(true);
            return;
        }
        //需要保存的数据
        page.putField("url",url);
        String title = html.xpath("//h1[@class='post_title']/text()").get();
        page.putField("title",title);
        String xString = html.xpath("//div[@class='post_info']/text()").get();
        String time = xString.replace("　来源:  北京  ", "");
        page.putField("time",time);
        String from = html.xpath("//div[@class='post_info']/a/text()").get();
        page.putField("from",from);
        String content = html.xpath("//div[@class='post_body']").get();
        page.putField("content",content);
    }

    @Override
    public Site getSite() {
        return site;
    }


    public static void main(String[] args) {
        Spider.create(new WangYiSpider())
                .addUrl("https://www.163.com/")
                .addPipeline(new Pipeline() {
                    @Override
                    public void process(ResultItems resultItems, Task task) {
                        Map<String, Object> all = resultItems.getAll();
                        System.out.println(all);
                    }
                })
                .addPipeline(new FilePipeline("d:\\webmagic\\"))
                .run();
    }
}
